In [1]:

    

%pylab inline

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from future.builtins import (bytes, str, open, super, range,
                      zip, round, input, int, pow, object)

if sys.version_info.major == 2:
    # in Python 2 cPickle is much faster than pickle but doesn't 
    # deal w/ unicode
    import cPickle as pickle
else:
    # Python 3 loads the faster pickle by default if it's available
    import pickle

# ---- Standard Libraries not included in pylab
import collections
import csv
import json
import os
import string
import sys
import time

# ---- Scientific Libraries
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import scipy as sp

# ---- GeoSpatial Libraries
import shapely

# ---- Extra Libraries for additional functionality
import singularity_logs


    

    




Populating the interactive namespace from numpy and matplotlib

In [2]:

    

# read in all the log entries from the log file
log_entries = []
with open("singularity.log", "r") as f:
    for line in f:
        log_entries.append(line.strip())
        
log_entries = json.loads("".join(log_entries))['logEntries']

# set up the singularity logs class
index_name = "singularity_logs"
sing = singularity_logs.SingularityLogs(index_name)

# delete any existing index
sing.delete_index()

# create a the index_name index and return ES results
print("Created {}\n{}\n".format(index_name, sing.create_index()))

# insert all log entries into index
for log in log_entries:
    sing.insert_log_entry(log)

time.sleep(3)
print(sing.count())


    

    




Created singularity_logs
{u'acknowledged': True}

{u'count': 1965, u'_shards': {u'successful': 5, u'failed': 0, u'total': 5}}

In [2]:

    

class Coord(object):
    """ Handles a latitude/longitude coordinate pair. """
    def __init__(self, *data):
        print(len(data))
        if len(data) == 1:
            coord_str = data[0][7:-1].split()
            self.lat = float(coord_str[1])
            self.lon = float(coord_str[0])
        elif len(data) == 2:
            self.lat = float(data[0])
            self.lon = float(data[1])
        else:
            raise 
        
    def __repr__(self):
        return "({:.5f}, {:.5f})".format(self.lat, self.lon)


    

In [3]:

    

class LogEntry(object):
    """ A data model/structure for a log entry. """
    def __init__(self, 
                 index=None,
                 log_level=None,
                 message_date=None,
                 app_id=None,
                 instance_id=None,
                 data=None):
        self.index = index
        self.log_level = log_level
        self.message_date = message_date
        self.app_id = app_id
        self.instance_id = instance_id
        self.data = data

    def __repr__(self):
        header = "=" * 70 + "\n"
        body1 = "index:\n{}\n\nlog_level:\n{}\n\nmessage_date:\n{}\n\n"\
                    .format(self.index, self.log_level, self.message_date)
        body2 = "app_id:\n{}\n\ninstance_id:\n{}\n\ndata:\n{}\n"\
                    .format(self.app_id, self.instance_id, self.data)
        footer = "-" * 70 + "\n"
        
        return header + body1 + body2 + footer


    

In [11]:

    

import elasticsearch
from elasticsearch import Elasticsearch

s = Elasticsearch(['http://search-01.ec2.internal:9200'])
results = s.search("singularity_logs")['hits']
print("Number of log entries:  {}\n"\
          .format(s.count("singularity_logs")['count']))

for entry in results['hits'][:4]:
    for element in entry:
        print(element, entry[element])
    print()


    

    




Number of log entries:  0

In [9]:

    

from elasticsearch import Elasticsearch
es = Elasticsearch(['http://search-01.ec2.internal:9200'])
es.indices.get_mapping("singularity_logs")


    

    
Out[9]:





{u'singularity_logs': {u'mappings': {u'log_entry': {u'properties': {u'data': {u'properties': {u'BBOX_WKT': {u'type': u'geo_shape'},
       u'BBOX_WKT2': {u'type': u'geo_shape'},
       u'RSV_IP_ADDRESS': {u'store': True, u'type': u'ip'},
       u'destIP': {u'store': True, u'type': u'ip'},
       u'point': {u'type': u'geo_point'}}}}}}}}

In [17]:

    

import shapely
polyshape = shapely.wkt.loads("POINT (78.259205566406 33.621632812500)")
geo_point = shapely.geometry.mapping(polyshape)


    

In [21]:

    

lon, lat = geo_point['coordinates']


    

In [22]:

    

print(lon, lat)


    

    




78.2592055664 33.6216328125

In [ ]: